#ifndef TOKZRGENTR_DATAS_INTERNAL_TOKENIZER_H
#define TOKZRGENTR_DATAS_INTERNAL_TOKENIZER_H

//# Tokenizer that the TokzrGentr gernerated would have 
//#	the follow structure.
//#	1.One char_set_table for hashing the ucs char.
//#	2.One connected_table for representing the dfa.
//#	3.One algoritms to run the dfa and tokenize the message.
//#	4.Other simple data structures for representing some neccessary info.
//#	5.Buffered IO and char encoding and decoding tools.(NOT included here)
#include"u16string_ref.h"
#include"char_set_table.h"
#include"tokenizer_datas.h"
#include<vector>
namespace TokzrGentr
{
	class internal_tokenizer
	{
	private:
		//#	internal_token representing 
		//#	the return value of this kind of tokenizer
		struct internal_token
		{
			std::u16string id;
			std::u16string content;
		};
	public:
		using token_type = internal_token;
		static char16_t end_of_token_tag[13];
		static char16_t end_of_token_content[1];
		std::shared_ptr<connected_table> _connected_table;
		std::shared_ptr<char_sets_table> _charset_table;
		//#using index to locate which token_type has been identified.
		std::vector<std::u16string> _token_types;
		
		std::u16string _str;
		unsigned _current_index;
		unsigned _recognized_toks_num = 0;
		bool _is_fail;
		bool _ignore_fail;
	public:
		internal_tokenizer(std::shared_ptr<connected_table> ct,
			std::shared_ptr<char_sets_table> cst,
			std::vector<std::u16string> token_types)
			:_connected_table(ct), _charset_table(cst),
			_token_types(token_types),_ignore_fail(true)
			,_str(u""),_is_fail(false),_current_index(0){}
		void SetStr(u16string_ref str)
		{
			_str = str.as_u16string();
			_is_fail = false;
			_current_index = 0;
		}
		void SetIgnoreFail(bool mode)
		{
			_ignore_fail = mode;
		}
		bool IsIgnoreFail() const
		{
			return _ignore_fail;
		}
		bool IsFail() const
		{
			return (!_ignore_fail&&_is_fail);
		}
		bool IsEOF() const
		{
			return _current_index >= _str.length();
		}
		token_type NextTok()
		{
			unsigned ch_index = _current_index;
			unsigned currenct_status = _connected_table->start_status();
			unsigned invalid_status = _connected_table->status_num();
			unsigned last_accepted_status = currenct_status;
			unsigned last_accepted_index = ch_index;
				
			//#	scanning the _str finding unaccepted transition
			while (currenct_status != invalid_status)
			{
				char_sets_table::ucs_type ch = _str[ch_index];
				currenct_status = 
					(*_connected_table)[currenct_status][(*_charset_table)[ch]];
				ch_index++;
				if (currenct_status != invalid_status &&
					_connected_table->get_mark(currenct_status) != 0)
				{
					last_accepted_status = currenct_status;
					last_accepted_index = ch_index;
				}
			}

			//# Found one accepted token
			if (last_accepted_index != _current_index)
			{
				auto id = _token_types[_connected_table->get_mark(last_accepted_status)];
				auto content = _str.substr(_current_index, last_accepted_index - _current_index);
				internal_token token = { id,content };
				_current_index = last_accepted_index ;
				_recognized_toks_num++;
				return token;
			}
			else
			{
				_is_fail = true;
				if (_ignore_fail && !IsEOF())
				{
					_current_index = _current_index + 1;
					return NextTok();
				}
				return { end_of_token_tag,end_of_token_content };
			}
		}
		unsigned RecognizedTokNum() const
		{
			return _recognized_toks_num;
		}
	};

	char16_t internal_tokenizer::end_of_token_tag[13] = u"END_OF_TOKEN";
	char16_t internal_tokenizer::end_of_token_content[1] = u"";
}
#endif // !TOKZRGENTR_DATAS_INTERNAL_TOKENIZER_H
